# original download of stockSymbols @ 8.7.2014
library(quantmod)

# get all symbols on US exchanges
symbols <- stockSymbols(c("AMEX", "NASDAQ", "NYSE"))
# clean dataset
## remove symbols with zero market cap
symbols.1 <- symbols[symbols$MarketCap > 0,]
## remove penny stocks
symbols.2 <- symbols.1[symbols.1$LastSale > 1,]
## remove stocks with IPOyear after 2000
symbols.3 <- symbols.2[is.na(symbols.2$IPOyear) | symbols.2$IPOyear <= 2000, ]
## remove stocks that are not assigned a sector
symbols.4 <- symbols.3[!is.na(symbols.3$Sector), ]
## order list by MarketCap
symbols.5 <- symbols.4[with(symbols.4, order(-MarketCap)),]
## remove stocks with very low MarketCap
symbols.6 <- symbols.5[1:round(nrow(symbols.5)*.9),]
save (list="symbols.6", file="symbols.6.RData")
syms <- symbols.6

# check if data for beginning of year 2000 is available
syms$fullset <- 0
for (i in 1:nrow(syms)) {
  try(getSymbols(syms[i,"Symbol"], from="2000-01-01", to="2000-01-10"), silent=TRUE)
  if(class(try(get(syms[i,"Symbol"])))[1] == "xts") {
    syms[i,"fullset"] <- 1
  } else {
    syms[i,"fullset"] <- 0
  }
}

# remove stocks where no data for beginning of 2000 is available
symbols.7 <- syms[syms$fullset > 0,]
save (list="symbols.7", file="symbols.7.RData")
syms <- symbols.7

# now retrieve price data (adjusted for splits and dividends) and create plots
syms$success <- 0
for (i in 1:nrow(syms)) {
  try(getSymbols(syms[i,"Symbol"], from="2000-01-01", to="2014-06-30", adjust=TRUE), silent=TRUE)
  if(class(try(get(syms[i,"Symbol"])))[1] == "xts") {
    syms[i,"success"] <- 1
    png(filename=paste0("/home/voellenk/plots/",syms[i,"Symbol"], ".png"))
    plot(Cl(get(syms[i,"Symbol"])), main=syms[i,"Symbol"], type="p", log="y", pch=20)
    dev.off()
  } else {
    syms[i,"success"] <- 0
  }
}

# filter out symbols that appear not suitable by optical inspection of plot
badsyms <- c("AAN", "ABT", "ACUR", "ADP", "AEGN", "AGX", "AI", "ALLB", "ALQA", "ALR",
             "AMCC", "AMRN", "ARCW", "ARDM", "ARIA", "ARQL", "ARTX", "ARWR",
             "ATRS", "AUBN", "AVNN", "AXLL", "AXP", "BAM", "BANF", "BIIB", "BOCH",
             "BOTA", "BRKL", "BSAC", "BWINA", "BWINB", "BXMT", "BXS", "BYBK",
             "CALL", "CBIN", "CBSH", "CDR", "CDZI", "CFFN", "CHNR", "CIA", "CLGX", "CMCT",
             "CMD", "CMT", "COB", "COP", "CPWR", "CQB", "CRAI", "CRESY", "CRRS",
             "CTIC", "CVC", "CVTI", "CY", "CYRN", "CYTR", "DEPO", "DF", "DGICB",
             "DLX", "DSCI", "DSS", "DUK", "EBIX", "EGAN", "EGBN", "ELLO", "ELRC", "ELX",
             "ELY", "EMKR", "ENG", "EPM", "ESI", "ESTE", "FBC", "FBSS", "FCNCA",
             "FHN", "FMBH", "FMER", "FNFG", "GILT", "GKNT", "GLDC", "GNCMA", "GRIF",
             "GTY", "HALL", "HAR", "HCBK", "HDSN", "HDY", "HILL", "HKTV", "HLS",
             "HNRG", "HRC", "HRTX", "HSH", "HSKA", "IACI", "ICAD", "ICGE", "IEP",
             "IESC", "IG", "IM", "INAP", "IPXL", "IRE", "IRS", "JAKK", "JDSU", "KELYB",
             "KINS", "KTOS", "LBAI", "LGND", "LION", "MAMS", "MATX", "MDR", "MDVN",
             "MEAS", "MLNK", "MRO", "MRVC", "MSI", "MSTR", "MXWL", "NBG", "NBIX",
             "NBS", "NC", "NEON", "NOC", "OKSB", "OMG", "ONTY", "OXBT", "OXGN",
             "PCTI", "PCYC", "PCYG", "PENN", "PFSW", "PGNX", "PHIIK", "PLUG", "PLX",
             "PLTX", "PNRG", "PNTR", "PVCT", "QADB", "RF", "RIG", "RNWK", "ROYL", "RPTP",
             "RTK", "SENEB", "SGU", "SNPS", "SRPT", "SSP", "STL", "STR", "TDS",
             "TTWO", "TYC", "ULBI", "UPI", "UPIP", "USEG", "UTHR", "VHC", "VIAS",
             "VICL", "VTNR", "VTSS", "WAC", "WAVX", "WETF", "XOMA", "YRCW")

syms$bad <- 0
for (i in 1:nrow(syms)) {
  if(syms[i,"Symbol"] %in% badsyms) {
    syms[i,"bad"] <-1
  }
}

symbols.8 <- syms[syms$bad<1,]
save (list="symbols.8", file="symbols.8.RData")
# this is the final data.frame
syms <- symbols.8[,1:8]

# now store the clean time series and the final syms data frame
varsToSave <- c("syms", syms$Symbol)
save (list=varsToSave, file="cleanStockData.RData")
